﻿open System
open System.Net
open System.Text.RegularExpressions

let fetchUrlSimple (url:string) = 
    let req = WebRequest.Create(url) 
    let response = req.GetResponse()
    use stream = response.GetResponseStream()
    use streamreader = new System.IO.StreamReader(stream)
    streamreader.ReadToEnd()

let topKWords (docs:string seq) K = 
    let separator = [|' '; '\r'; '\n'; '-'; '.'; ',' ; '\t'; '!'; '?'; '\''; ';'; '/' |]
    let stopwords = Set("a,able,about,across,after,all,almost,also,am,among,an,and,any,are,as,at,be,because,been,but,by,can,cannot,could,dear,did,do,does,either,else,ever,every,for,from,get,got,had,has,have,he,her,hers,him,his,how,however,i,if,in,into,is,it,its,just,least,let,like,likely,may,me,might,most,must,my,neither,no,nor,not,of,off,often,on,only,or,other,our,own,rather,said,say,says,she,should,since,so,some,than,that,the,their,them,then,there,these,they,this,tis,to,too,twas,us,wants,was,we,were,what,when,where,which,while,who,whom,why,will,with,would,yet,you,your".Split ',')

    docs
    |> Seq.map (fun doc ->
        doc.Split(separator, StringSplitOptions.RemoveEmptyEntries)
        |> Seq.map (fun word -> word.ToLower())
        |> Seq.filter (fun word -> not (stopwords.Contains(word)))
        )
    |> Seq.concat
    |> Seq.groupBy (fun x->x)
    |> Seq.map (fun (word, wordSeq) -> (word, wordSeq |> Seq.length))
    |> Seq.sortBy (fun (_, wordCnt) -> - wordCnt)
    |> Seq.take K
    |> Seq.toList

let rawpage = fetchUrlSimple @"http://channel9.msdn.com/Events/MIX/MIX11" 
let titles, abstracts =
    let page = Regex.Replace(rawpage, "&#?[a-z0-9]+;", " ")
    let titleMatches = Regex.Matches(page, "class=\"title\">(.*?)</a>")
    let abstractMatches = Regex.Matches(page.Replace("\n"," "), "class=\"description\">(.*?)</div>") 
    let matchesToSeq (matches: MatchCollection) = 
        seq {
            for m in matches do
                yield m.Groups.[1].Value
        }
    matchesToSeq titleMatches, matchesToSeq abstractMatches


topKWords titles 20
topKWords abstracts 20
